import requests
from lxml import etree
from celery import Celery
app = Celery('tasks', broker='redis://localhost:6379/1',backend='redis://localhost:6379/2')
@app.task
def first():
    print("你好这是我第一个异步任务")
    fiction_list = []
    url = 'http://www.xbiquge.la/xiaoshuodaquan/'
    response = requests.get(url=url)
    html = etree.HTML(response.text)
    all_fiction = html.xpath("//div[@class='novellist']/ul/li/a")
    for i in all_fiction:
        fiction = i.xpath("./text()")[0]
        fiction_href = i.xpath("./@href")[0]
        fiction_list.append({"fiction_name": fiction, "fiction_url": fiction_href})
    print('111111111111111111111', fiction_list)
    return fiction_list

def get_all_section():
    section_list = []
    fiction_list = first()
    for i in fiction_list:
        url = i['fiction_url']
        # url = 'http://www.xbiquge.la/15/15409/'
        response = requests.get(url=url)
        response.encoding = response.apparent_encoding
        html = etree.HTML(response.text)
        all_section = html.xpath("//div[@id='list']/dl/dd/a")
        for i in all_section:
            section = i.xpath("./text()")[0]
            section_href = 'http://www.xbiquge.la' + i.xpath("./@href")[0]
            section_list.append({"fiction": section, "section_url": section_href})
        print('2222222222222222222222222', section_list)
    return section_list

def get_all_content():
    content_list = []
    section_list = get_all_section()
    for i in section_list:
        url = i['section_url']
        print(url)
        response = requests.get(url=url)
        response.encoding = response.apparent_encoding
        html = etree.HTML(response.text)
        content = html.xpath("string(//div[@id='content'])")
        content_list.append({"content": content})
        print(content_list)
    print('33333333333333333333', content_list)
    return content_list

@app.task
def hanshu():
    a = get_all_content()
    # b = get_all_section()